from concurrent.futures import ThreadPoolExecutor  # 线程池
import threading
from threading import Thread
import time
import queue
import json
import re
import string
import requests

'''
# url列表文件分成多个文件
with open("get_data/url_provincescore.txt", encoding='utf-8') as s:
    with open("get_data/url_provincescore1.txt","a+", encoding='utf-8')as a:
        with open("get_data/url_provincescore2.txt","a+", encoding='utf-8')as b:
            i=0
            line=s.readline()
            while i<300000:
                a.write(line)
                i=i+1
                line = s.readline()
            while i>299999 and i<679502:
                b.write(line)
                i=i+1
                line = s.readline()

'''

class TestClass():
    def __init__(self,links, result_queue): #参数分别为url列表和队列

        # 定义10个线程的线程池用于请求数据
        self.thread_pool1 = ThreadPoolExecutor(max_workers=10)
        self.links = links
        self.result_queue= result_queue

    def main_logic(self):

        for link in self.links:
            self.thread_pool1.submit(self.do_request, link)

    def do_request(self, link):

        #分出学校id字段
        # divide = re.split('/', link)
        # school_id=divide[5]

        data = requests.get(link.rstrip("\n"))
        print(link)

        if(data.text!="\"\""):
            # self.result_queue.put(link.rstrip("\n"))
            self.result_queue.put(data.text)



def write_to_file(q,file):

    while q.empty() != True:
        file.write(str(q.get())+"\n")

    print("打印完毕")

if __name__ == "__main__":

    school_list = []
    school_list1 = []
    year_list = []
    province_list = []
    type_list = []
    url_list = []

    # # 读取”学校id名称对应.txt“获取学校id列表
    # with open("get_data/学校id名称对应.txt", encoding='utf-8')as s:
    #     line = s.readline()
    #     while line:
    #         divide = re.split(',', line)
    #         school_list.append(int(divide[1]))
    #         line = s.readline()
    #
    # # 读取”年份省份type列表.txt“获取相应数据
    # with open("get_data/年份省份type列表.txt", encoding='utf-8')as f:
    #     line = f.readline()
    #     while line:
    #         divide = re.split(',', line)
    #         school_list1.append(int(divide[0]))
    #         year_list.append(int(divide[1]))
    #         province_list.append(int(divide[2]))
    #         type_list.append(int(divide[3]))
    #         line =f.readline()

    # 读取”学校年份省份type列表.txt“获取相应数据
    with open("get_data/学校年份省份type列表.txt", encoding='utf-8')as f:
        line = f.readline()
        while line:
            divide = re.split(',', line)
            school_list1.append(int(divide[1]))
            year_list.append(int(divide[2]))
            province_list.append(int(divide[3]))
            type_list.append(int(divide[4]))
            line = f.readline()

    print("列表均读取完成")

    length = len(year_list)

    #分别为”各省分数线“、”招生计划“、”专业分数线目录“、“专业分数线详情页面”的url
    link1=["https://static-data.eol.cn/www/2.0/school/{}/dic/provincescore.json".format(i) for i in school_list]
    link2=["https://static-data.eol.cn/www/2.0/school/{}/dic/specialplan.json".format(i) for i in school_list]
    link3=["https://static-data.eol.cn/www/school/{}/dic/specialscore.json".format(i) for i in school_list]

    urlProvinceLine_start = "https://static-data.eol.cn/www/2.0/schoolspecialindex"
    urlProvinceLine_end = ".json"

    urlProvinceScore_start = "https://static-data.eol.cn/www/2.0/schoolprovinceindex/detial"
    urlProvinceScore_end = ".json"

    q = queue.Queue()
    file = open("get_data/各省分数线爬取数据.txt", "a+")  # 要写入的文件

    print("建立url列表中...")

    # f = open("get_data/url_provincescore.txt", "a+")
    # # 第一次建立url列表时需要使用
    # for x in range(0,length):
    #     for page in range(1,3):
    #         url = urlProvinceScore_start + "/" + str(school_list1[x]) + "/" + str(province_list[x]) + "/" + str(type_list[x]) + "/" + str(page) + urlProvinceScore_end
    #         f.write(url+"\n")

    with open("get_data/url_provincescore1.txt","r", encoding='utf-8')as f: #要爬取的url列表文件
        line = f.readline()
        while line:
            url_list.append(line)
            line = f.readline()

    print("列表建立完毕")

    obj = TestClass(url_list, q) #第一个参数link为要爬取的url列表
    obj.main_logic()
    obj.thread_pool1.shutdown()

    # 写文件线程
    t = Thread(target=write_to_file, args=(q, file))
    t.start()


'''
# 读取”专业分数线.txt“获取学校省份和type对应的“年份省份type列表.txt”
with open("get_data/年份省份type列表.txt","a+", encoding='utf-8') as s:
    with open("get_data/专业分数线.txt", encoding='utf-8')as f:
        line = f.readline()
        school = line.rstrip('\n')
        line = f.readline()
        while line:
            body = json.loads(line)
            if isinstance(body, dict):
                length1 = len(body["data"]) #年份列表
                for x in range (0,length1):
                    year = body["data"][x]["year"]
                    length2 = len(body["data"][x]["province"]) #省份列表
                    for y in range(0, length2):
                        province = body["data"][x]["province"][y]["pid"]
                        length3 = len(body["data"][x]["province"][y]["type"])
                        for z in range(0,length3):
                            type = body["data"][x]["province"][y]["type"][z]
                            s.write(school+","+str(year)+","+str(province)+","+str(type)+"\n")

            line = f.readline()
            school = line.rstrip('\n')
            line = f.readline()
'''
'''
# 读取”各省分数线.txt“获取学校省份和type对应的“学校年份省份type列表.txt”
with open("get_data/学校年份省份type列表.txt","a+", encoding='utf-8') as s:
    with open("get_data/各省分数线.txt", encoding='utf-8')as a:
        with open("get_data/学校id名称对应.txt", encoding='utf-8')as b:
            line1 = b.readline()
            school = line1.rstrip('\n')
            line2 = a.readline()
            while line2:
                body = json.loads(line2)
                if isinstance(body, dict):
                    length1 = len(body["data"]["data"]) #年份列表
                    for x in range (0,length1):
                        year = body["data"]["data"][x]["year"]
                        length2 = len(body["data"]["data"][x]["province"]) #省份列表
                        for y in range(0, length2):
                            province = body["data"]["data"][x]["province"][y]["pid"]
                            length3 = len(body["data"]["data"][x]["province"][y]["type"])
                            for z in range(0,length3):
                                type = body["data"]["data"][x]["province"][y]["type"][z]
                                s.write(school+","+str(year)+","+str(province)+","+str(type)+"\n")

                line1 = b.readline()
                school = line1.rstrip('\n')
                line2 = a.readline()
'''
'''
#读取”学校id名称对应.txt“获取学校id列表
school_list=[]
with open("get_data/学校id名称对应.txt",encoding='utf-8')as s:
    line = s.readline()
    while line:
        divide = re.split(',',line)
        school_list.append(int(divide[1]))
        line=s.readline()

#检查学校列表是否正确
# length= len(school_list)
# print(length)
'''

'''
#按照学校列表请求获取学校信息统计存入“学校信息统计.txt”
links = ["https://static-data.eol.cn/www/school/{}/pc_jobdetail.json".format(i) for i in school_list]
with open("get_data/学校信息统计.txt", "a+",encoding='utf-8') as f:
    for i in links:
        data = requests.get(i)  #data.text是json全部信息
        body = json.loads(data.text)  #转换为json格式
        if isinstance(body, dict):
            if(data.text!="\"\""):
                print(i)
                f.write(data.text+"\n")
'''

'''
#读取“学校信息统计.txt”获取对应学校就业率、升学率和出国率存入“学校毕业去向统计表.txt”
with open("get_data/学校毕业去向统计表.txt","a+",encoding='utf-8')as s:
    with open("get_data/学校信息统计.txt",encoding='utf-8')as f:
        s.write("job" + "," + "postgraduate" + "," + "abroad" + "," + "female_num" + "," + "female_rate" + "," + "men_num" + "," + "men_rate" + "\n")
        line = f.readline()#适合读取大文件
        while line:
            body = json.loads(line)
            if isinstance(body, dict):
                if isinstance(body["jobrate"],dict):

                    #就业率
                    if isinstance(body["jobrate"]["job"],dict):
                        if("1" in body["jobrate"]["job"]):
                            job = body["jobrate"]["job"]["1"]
                        else:
                            job = str(0)
                    else:
                        job = str(0)

                    #国内升学率
                    if isinstance(body["jobrate"]["postgraduate"],dict):
                        if("1" in body["jobrate"]["postgraduate"]):
                            postgraduate = body["jobrate"]["postgraduate"]["1"]
                        else:
                            postgraduate = str(0.00)
                    else:
                        postgraduate = str(0.00)

                    #出国率
                    if isinstance(body["jobrate"]["abroad"],dict):
                        if("1" in body["jobrate"]["abroad"]):
                            abroad = body["jobrate"]["abroad"]["1"]
                        else:
                            abroad = str(0.00)
                    else:
                        abroad = str(0.00)

                else:
                    job=str(0.00)
                    postgraduate = str(0.00)
                    abroad = str(0.00)

                if isinstance(body["gradute"],list):

                    #女生人数
                    if ("female_num" in body["gradute"][0]):
                        female_num = body["gradute"][0]["female_num"]
                    else:
                        female_num = str(0)

                    # 女生比例
                    if ("female_rate" in body["gradute"][0]):
                        female_rate = body["gradute"][0]["female_rate"]
                    else:
                        female_rate = str(0.00)

                    # 男生人数
                    if ("men_num" in body["gradute"][0]):
                        men_num = body["gradute"][0]["men_num"]
                    else:
                        men_num = str(0)

                    # 男生比例
                    if ("men_rate" in body["gradute"][0]):
                        men_rate = body["gradute"][0]["men_rate"]
                    else:
                        men_rate = str(0.00)

                else:
                    female_num = str(0)
                    female_rate = str(0.00)
                    men_num = str(0)
                    men_rate = str(0.00)

                s.write(job+","+postgraduate+","+abroad+","+female_num+","+female_rate+","+men_num+","+men_rate+"\n")

            else:
                s.write(str(0.00)+","+str(0.00)+","+str(0.00)+","+str(0)+","+str(0.00)+","+str(0)+","+str(0.00)+"\n")

            line=f.readline()

'''

'''
#请求0~3600范围内的url获取学校info存入“学校info.txt”
links = ["https://static-data.eol.cn/www/2.0/school/{}/info.json".format(i) for i in range(0, 3600)]
with open("get_data/学校info.txt", "a+",encoding='utf-8') as f:
    for i in links:
        data = requests.get(i)  #data.text是json全部信息
        body = json.loads(data.text)  #转换为json格式
        if isinstance(body, dict):
            if(data.text!="\"\""):
                print(body)
                f.write(data.text+"\n")
'''

'''
#读取“学校info.txt”获取对应学校id和name存入“学校id名称对应.txt”
with open("get_data/学校id名称对应.txt","a+",encoding='utf-8')as s:
    with open("get_data/学校info.txt",encoding='utf-8')as f:
        line = f.readline()#适合读取大文件
        while line:
            body = json.loads(line)
            print(body)
            if isinstance(body, dict):
                name = body["data"]["name"]
                school_id = body["data"]["school_id"]
                # province_name = body["data"]["province_name"]
                # num_master = body["data"]["num_master"]
                # num_doctor = body["data"]["num_doctor"]
                # num_subject = body["data"]["num_subject"]
                # num_lab = body["data"]["num_lab"]
                s.write(name + "," + school_id+ "\n")
            line=f.readline()
'''